package Crawler;

import java.io.File;
import java.io.IOException;
//import java.net.UnknownHostException;
import java.util.ArrayList;

import org.apache.commons.io.FileUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import com.google.common.base.Joiner;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.MongoClient;

public class Spider {

    // 连接Mongodb
    public static void writeDataIntoMongo(BasicDBObject document) {
        MongoClient mongo;
        try {
            mongo = new MongoClient("222.27.227.24", 27017);
            DB db = mongo.getDB("Attention");
            DBCollection table = db.getCollection("Following");
            // save与insert都能向mongodb中插入数据
            // save原对象不存在可以创建，存在可更新
            // insert可以一次性插入一整表不需要遍历
            table.save(document);
            // table.insert(document);
        } catch (Exception e) {
            e.printStackTrace();
        }

        // } catch (UnknownHostException e) {
        //  //TODO Auto-generated catch block
        // e.printStackTrace();
        // }

    }

    // 获取文件中的所有数据
    public static ArrayList getUidList(String fileName) {
        //
        ArrayList uidList = new ArrayList();
        try {
            for (String line : FileUtils.readLines(new File(fileName))) {
                uidList.add(line);
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return uidList;
    }

    //数据爬取以及数据的存入
    public static void crawler_and_toMongo(String uid, int following) {
        // 微博关注页数中最大展示关注数一页32个
        float total_number = (float) (following * 1.0 / 32 + 3);
        ArrayList following_list = new ArrayList();

        for (int i = 1; i < total_number; i++) {
            System.out.println("The current uid is " + uid + " and the page number is " + i);
            String following_url = "http://tw.weibo.com/" + uid + "/follow/p/" + i;
            // Jsoup 是一个 Java 的开源HTML解析器，可直接解析某个URL地址、HTML文本内容
            Connection con = Jsoup.connect("http://tw.weibo.com/api/user/follow");
            con.data("uid", uid);
            con.data("page", String.valueOf(i));
            con.data("currentuid", "1991953413");
            con.data("page_size", "32");
            // 请求头
            con.header("Cookie",
                    "SINAGLOBAL=9711952802808.682.1499668084041; __gads=ID=130480cbf75a5ad6:T=1500258878:S=ALNI_MZRcFnJveJoxtD4TBbJNbD__KcD-A; UM_distinctid=15d4e658b271f5-0573ee5fbac68c-1b1d7751-1fa400-15d4e658b28944; wvr=6; UOR=,,www.shejidaren.com; _ga=GA1.2.84290483.1500258858; _gid=GA1.2.952496907.1501988163; crtg_rta=; SSOLoginState=1502013392; SCF=Agl4g4wcoIanRFGqnVTCrYVRKcGcCSGCXik1olhn6h9oPJpHfvjbBZyqPuHU-ibuQXyHNclynoHBANDkf_WRtWc.; SUB=_2A250gpeCDeRhGedH4lMY9S3Nzz-IHXVX-Y5KrDV8PUNbmtBeLUWjkW9XZkSwKQAYTJkZTvK42M-ekE_Hnw..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWEbrh.i5bvbnSvvbQHi2M35JpX5KMhUgL.Fo241K24SKepShe2dJLoIpjLxKML1KBLBo5LxK-LB--LB-2LxK-L1h-L1h2t; SUHB=0LcjYQm8zbqhfQ; ALF=1533549388; _s_tentry=-; Apache=9574205823039.291.1502013327944; ULV=1502013327950:6:1:2:9574205823039.291.1502013327944:1501421673576");
            // con.header("Referer","http://tw.weibo.com/libingbing/follow/p/12");
            con.header("Referer", following_url);
            Document doc;
            try {
                doc = con.post();
                for (Element e : doc.select("[class =picBox]")) {
                    String following_uid = e.getElementsByTag("div").attr("uid");
                    // System.out.println(following_uid);
                    following_list.add(following_uid);
                }
                // 请求时间在5-15中随机抽取一个数字
                timeDelay(5, 15);
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                // e1.printStackTrace();
                timeDelay(30, 35);
                continue;
            }
            // System.out.println(doc.select("[class =name]").size());
            // for (Element e: doc.select("[class =name]")) {
            // System.out.println(e.text()+'\t'+e.getElementsByTag("h3").get(0).getElementsByTag("a").attr("href"));
            // System.out.println(doc.select("[class =picBox]").size());
        }
        String uids = Joiner.on(",").join(following_list);
        // System.out.println("The uids is "+uids);
        BasicDBObject document = new BasicDBObject();
        document.put("_id", uid);
        document.put("following", uids);

        writeDataIntoMongo(document);
    }


    // 随机获取时间段
    public static void timeDelay(float min, float max) {
        int random = (int) (max * Math.random() + min);
        try {
            Thread.sleep(random * 1000);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    // COOKIES文件。。。貌似没有用到
    public static ArrayList getCookies() {
        ArrayList cookies = new ArrayList();
        try {
            for (String line : FileUtils.readLines(new File("D:/Data/Zoufan/MetaData/cookies.txt"))) {
                cookies.add(line);
            }
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        return cookies;
    }

    // 主函数
    public static void main(String[] args) throws IOException {
        // TODO Auto-generated method stub
        String fileName = "C:/WEIBO/414+UID.txt";
        ArrayList lineList = new ArrayList();
        // 读取文件中的一行 uid,connum
        lineList = getUidList(fileName);
        // 遍历这个字符串
        for (int i = 0; i < lineList.size(); i++) {

            // 将文件中的每一行变成字符串类型
            String line = lineList.get(i).toString();
            // 字符串分割 java.lang包中有String.split()方法，返回是一个数组
            String[] parts = line.split(",");
            // uids
            String uid = parts[0];
            // connums   Integer.parseInt输出一个十进制的数字，radix表述逗号前面的是什么进制的数字，无论参数是什么进制输出都是十进制
            int following = Integer.parseInt(parts[1], 10);
            //
            crawler_and_toMongo(uid, following);
        }
    }

}
